# CS587 - Assignment 5 (Part II)
## Tracking Issues created for Repos on GitHub
## Plot and Forcaste Issues
### Deliverables:
- Submit a single ZIP file for both Part 1 & Part 2 that has the folowing items:
1. Your PDF document for the analysis report.
2. Your IPYNB script with the name GitHub_Repos_Issues_Forecasting.ipynb that has your source code and output for the requirements listed
2. Your HTML/PDF document that has your source code and output for the requirements listed
## REQUIREMENT 1
Use Python/GitHub API to retrieve Issues/Repos information of the past 3 years for the following repositories:
- https://github.com/angular/angular
- https://github.com/angular/material
- https://github.com/angular/angular-cli
- https://github.com/SebastianM/angular-google-maps
- https://github.com/d3/d3
In [ ]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import github3
import json
# Get an API key for GitHub and set it as GITHUB_TOKEN
# Here is the URL to guide you on how to generate your GITHUB_TOKEN
# https://help.github.com/articles/creating-an-access-token-for-command-line-use/
# create your GitHub personal access tokens from https://github.com/settings/tokens
GITHUB_TOKEN = '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' # 'ADD-YOUR-GitHub-Personal-Token-HERE'
gh = github3.login(token=GITHUB_TOKEN)
from tqdm import tqdm
import dateutil
import datetime
import time
def getRepoIssue(repo):
print(repo)
ORG = repo[0]
REPO = repo[1]
FILENAME_ISSUES = ORG + REPO+ 'issues.json'
inputFile = open('./{}'.format(FILENAME_ISSUES), 'w')
today = datetime.date.today()
for i in tqdm(range(36)):#36
last_month = today + dateutil.relativedelta.relativedelta(months = -1)
types = 'type:issue'
repo = 'repo:angular/angular'
ranges = 'created:'+str(last_month)+'..'+str(today)
search_query = types + ' ' + repo + ' ' + ranges
for issue in gh.search_issues(search_query):
label_name=[]
data={}
current_issue = issue.as_json()
current_issue = json.loads(current_issue)
data['issue_number']=current_issue["number"] # Get issue number
data['created_at']= current_issue["created_at"][0:10] # Get created date of issue
if current_issue["closed_at"] == None:
data['closed_at']= current_issue["closed_at"]
else:
data['closed_at']= current_issue["closed_at"][0:10] # Get closed date of issue
for label in current_issue["labels"]:
label_name.append(label["name"]) # Get label name of issue
data['labels']= label_name
data['State'] = current_issue["state"] # It gives state of issue like closed or open
data['Author'] = current_issue["user"]["login"] # Get Author of issue
out=json.dumps(data) # save this all information to a JSON file
inputFile.write(out+ '\n')
today = last_month
time.sleep(10)
inputFile.close()
print('Done')
repolist = [('angular' ,'angular'),('angular' ,'material'),('angular' ,'angular-cli'),('SebastianM','angular-google-maps'),('d3','d3')]
for repo in repolist:
getRepoIssue(repo)
In [ ]:
import os
In [459]:
repos = [x for x in os.listdir('./issues') if x[0] != '.']
dfs = pd.DataFrame()
for repo in repos:
list_of_issues_dict_data = [json.loads(line) for line in open('./issues/{}'.format(repo))]
issues_df = pd.DataFrame(list_of_issues_dict_data)
issues_df['repo'] = repo.split('.')[0]
dfs = dfs.append(issues_df,ignore_index=True)
In [ ]:
import warnings
warnings.filterwarnings('ignore')
## REQUIREMENT 2
- A line chart to plot the issues for every repository
In [461]:
dfs.groupby(['repo']).count()[['issue_number']].plot(figsize=(25,10))
Out[461]:
## REQUIREMENT 3
- A bar chart to plot the issues created for every month for every repository
In [462]:
dfs['create_month'] = dfs['created_at'].map(lambda x: int(x.split('-')[1]))
In [463]:
dfs.groupby(['create_month','repo']).create_month.count().unstack().plot(kind='bar',stacked=False,figsize=(25,10))
Out[463]:
## REQUIREMENT 4
- A bar chart to plot the starts for every repository
In [ ]:
## REQUIREMENT 5
- A bar chart to plot the forks for every repository
## REQUIREMENT 6
- A bar chart to plot the issues closed for every week for every repository
In [464]:
dfs6= dfs[dfs['closed_at'].notna()]
In [465]:
from datetime import datetime
dfs6['closed_weekday'] = [datetime.strptime(x,'%Y-%m-%d').weekday() for x in dfs6['closed_at']]
In [466]:
dfs6.groupby(['closed_weekday','repo']).closed_weekday.count()\
.unstack().plot(kind='bar',stacked=False,figsize=(25,10))
Out[466]:
## REQUIREMENT 7
- A stack-bar Chart to plot the created and closed issues for every repository
In [ ]:
#Add your code for requirement 7 in this cell
In [467]:
dfs.groupby(['repo']).count()[['closed_at','created_at']].plot(kind='bar',stacked=True,figsize=(25,10))
Out[467]:
## REQUIREMENT 8 (FACEBOOK/PROPHET)
- Use Facebook/Prophet package to forecast the following for every repository
In [ ]:
import prophet
from prophet import Prophet
#### REQ8-1). The day of the week maximum number of issues created
In [470]:
def r1(reponame):
df = dfs[dfs['repo'] ==reponame]
df = df.groupby('created_at')['created_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf
pdf['ds_new'] = pd.to_datetime(pdf['ds']) - pd.to_timedelta(7, unit='d')
df_weekly_max = pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='W-MON')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_orig','y']])
df_final = pd.DataFrame(['ds','y'])
df_final = df_weekly_max[['ds_orig','y']]
df_final.columns = ['ds','y']
m = Prophet()
m.fit(df_final)
future = m.make_future_dataframe(periods=365) # Forcast for 1 year
forecast = m.predict(future)
fig1 = m.plot(forecast)
In [471]:
for i in range(5):
r1(dfs['repo'].unique()[i])
#### REQ8-2). The day of the week maximum number of issues closed
In [472]:
def r2(reponame):
df = dfs6[dfs6['repo'] ==reponame]
df = df.groupby('closed_at')['closed_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds']) - pd.to_timedelta(7, unit='d')
df_weekly_max = pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='W-MON')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_orig','y']])
df_final = pd.DataFrame(['ds','y'])
df_final = df_weekly_max[['ds_orig','y']]
df_final.columns = ['ds','y']
m = Prophet()
m.fit(df_final)
future = m.make_future_dataframe(periods=365) # Forcast for 1 year
forecast = m.predict(future)
fig1 = m.plot(forecast)
In [474]:
for i in range(5):
print(dfs['repo'].unique()[i])
r2(dfs['repo'].unique()[i])
In [ ]:
#### REQ8-3). The month of the year that has maximum number of issues closed
In [ ]:
In [475]:
def r3(reponame):
df = dfs6[dfs6['repo'] ==reponame]
df['closed_at'] = df['closed_at'].map(lambda x: '-'.join(x.split('-')[:-1]+['01']))
df = df.groupby('closed_at')['closed_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds'])
# pdf['ds_new'] = pd.to_datetime(pdf['ds']) - pd.to_timedelta(1, unit='m')
df_weekly_max = pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='Y')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_orig','y']])
df_final = pd.DataFrame(['ds','y'])
df_final = df_weekly_max[['ds_orig','y']]
df_final.columns = ['ds','y']
# df_final['ds'] = df_final['ds'].map(lambda x: '-'.join(x.split('-')[:-1]))
m = Prophet()
m.fit(df_final)
future = m.make_future_dataframe(periods=12) # Forcast for 1 year
forecast = m.predict(future)
fig1 = m.plot(forecast)
In [476]:
for i in range(5):
print(dfs['repo'].unique()[i])
r3(dfs['repo'].unique()[i])
#### REQ8-4). Plot the created issues forecast
In [ ]:
def r4(reponame):
df = dfs[dfs['repo'] ==reponame]
df = df.groupby('created_at')['created_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds'])
df_final = pd.DataFrame(['ds','y'])
df_final = pdf[['ds_orig','y']]
df_final.columns = ['ds','y']
m = Prophet()
m.fit(df_final)
future = m.make_future_dataframe(periods=365) # Forcast for 1 year
forecast = m.predict(future)
fig1 = m.plot(forecast)
In [477]:
for i in range(5):
print(dfs['repo'].unique()[i])
r4(dfs['repo'].unique()[i])
#### REQ8-5). Plot the closed issues forecast
In [480]:
def r5(reponame):
df = dfs6[dfs6['repo'] ==reponame]
df = df.groupby('closed_at')['closed_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds'])
df_final = pd.DataFrame(['ds','y'])
df_final = pdf[['ds_orig','y']]
df_final.columns = ['ds','y']
m = Prophet()
m.fit(df_final)
future = m.make_future_dataframe(periods=365) # Forcast for 1 year
forecast = m.predict(future)
fig1 = m.plot(forecast)
In [481]:
for i in range(5):
print(dfs['repo'].unique()[i])
r5(dfs['repo'].unique()[i])
#### REQ8-6). Plot the pulls forecast
#### REQ8-7). Plot the commits forecast
#### REQ8-8). Plot the branches forecast
#### REQ8-9). Plot the contributors forecast
#### REQ8-10). Plot the releases forecast
## REQUIREMENT 8 (TENSORFLOW/KERAS LSTM)
- Use TensorFlow/Keras LSTM package to forecast the following for every repository
In [ ]:
from tensorflow import keras
In [ ]:
def plot_and_train(tensor_Created):
xtrain = tensor_Created.iloc[0:int(len(tensor_Created)*0.8)]['time']\
.to_numpy().reshape(int(len(tensor_Created)*0.8),1,1)
xtrain.shape
ytrain = tensor_Created.iloc[0:int(len(tensor_Created)*0.8)]['value']\
.astype('float32')\
.to_numpy()
ytrain.shape
model = keras.Sequential()
model.add(keras.layers.LSTM(
units=128,input_shape=(1,1)
))
model.add(keras.layers.Dropout(0.2))
model.add(keras.layers.Dense(units=1))
model.compile(
loss= 'mean_squared_error',optimizer='adam')
model.fit(xtrain,ytrain,epochs=30)
xtest = tensor_Created.iloc[int(len(tensor_Created)*0.8):]['time']\
.to_numpy().reshape(len(tensor_Created) - int(len(tensor_Created)*0.8),1,1)
ytest = tensor_Created.iloc[int(len(tensor_Created)*0.8):]['value']\
.astype('float32')\
.to_numpy()
ypred = model.predict(xtest)
fig,axs = plt.subplots(1,1,figsize=(20,8))
axs.plot([x for x in range(0,len(ytrain))],ytrain,'g',label='history')
axs.plot([x for x in range(len(ytrain),len(ytrain)+len(ytest))]
,ytest,marker='.',label='true')
axs.plot([x for x in range(len(ytrain),len(ytrain)+len(ytest))]
,ypred,'r',label='prediction')
plt.show()
#### REQ8-1). The day of the week maximum number of issues created
In [482]:
def k1(reponame):
df = dfs[dfs['repo'] ==reponame]
df = df.groupby('created_at')['created_at']
df_new = df.describe()
dfnew1 = pd.Series(df_new['top'], name='ds_original')
df_new = pd.concat([df_new, dfnew1], axis=1)
datafrm_pdf = pd.DataFrame(['ds','ds_original','y'])
datafrm_pdf = df_new[['top','ds_original','count']]
datafrm_pdf.columns = ['ds','ds_original','y']
datafrm_pdf['ds_new'] = pd.to_datetime(datafrm_pdf['ds']) - pd.to_timedelta(7, unit='d')
df_weekly_maximum = datafrm_pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='W-MON')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_original','y']])
df_created_output = pd.DataFrame(['ds','y'])
df_created_output = df_weekly_maximum[['ds_original','y']]
df_created_output.columns = ['ds','y']
tensor_Created = df_created_output
tensor_Created = tensor_Created[['ds','y']]
df = pd.DataFrame(tensor_Created)
tensor_Created.rename(columns={'ds':'timestamp'}, inplace=True)
tensor_Created.rename(columns={'y':'value'}, inplace=True)
# print(tensor_Created)
firstDay = min(pd.to_datetime(tensor_Created['timestamp']))
tensor_Created['time'] = [float(x.days) for x in \
[x - firstDay for x in pd.to_datetime(tensor_Created['timestamp'])]]
return tensor_Created
In [483]:
for i in range(5):
plot_and_train(k1(dfs['repo'].unique()[i]))
#### REQ8-2). The day of the week maximum number of issues closed
In [ ]:
def k2(reponame):
df = dfs6[dfs6['repo'] ==reponame]
df = df.groupby('closed_at')['closed_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf
pdf['ds_new'] = pd.to_datetime(pdf['ds']) - pd.to_timedelta(7, unit='d')
df_weekly_max = pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='W-MON')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_orig','y']])
# print(df_weekly_max)
df_final = pd.DataFrame(['ds','y'])
df_final = df_weekly_max[['ds_orig','y']]
df_final.columns = ['timestamp','value']
firstDay = min(pd.to_datetime(df_final['timestamp']))
df_final['time'] = [float(x.days) for x in \
[x - firstDay for x in pd.to_datetime(df_final['timestamp'])]]
return df_final
In [486]:
for i in range(5):
plot_and_train(k2(dfs['repo'].unique()[i]))
#### REQ8-3). The month of the year that has maximum number of issues closed
In [ ]:
def k3(reponame):
df = dfs6[dfs6['repo'] ==reponame]
df['closed_at'] = df['closed_at'].map(lambda x: '-'.join(x.split('-')[:-1]+['01']))
df = df.groupby('closed_at')['closed_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds'])
# pdf['ds_new'] = pd.to_datetime(pdf['ds']) - pd.to_timedelta(1, unit='m')
df_weekly_max = pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='Y')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_orig','y']])
print(df_weekly_max)
df_final = pd.DataFrame(['ds','y'])
df_final = df_weekly_max[['ds_orig','y']]
df_final.columns = ['ds','y']
# df_final['ds'] = df_final['ds'].map(lambda x: '-'.join(x.split('-')[:-1]))
df_final.columns = ['timestamp','value']
firstDay = min(pd.to_datetime(df_final['timestamp']))
df_final['time'] = [float(x.days/30) for x in \
[x - firstDay for x in pd.to_datetime(df_final['timestamp'])]]
return df_final
In [488]:
for i in range(5):
plot_and_train(k3(dfs['repo'].unique()[i]))
#### REQ8-4). Plot the created issues forecast
In [ ]:
def k4(reponame):
df = dfs[dfs['repo'] ==reponame]
df = df.groupby('created_at')['created_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds'])
df_final = pd.DataFrame(['ds','y'])
df_final = pdf[['ds_orig','y']]
df_final.columns = ['timestamp','value']
firstDay = min(pd.to_datetime(df_final['timestamp']))
df_final['time'] = [float(x.days) for x in \
[x - firstDay for x in pd.to_datetime(df_final['timestamp'])]]
return df_final
In [490]:
for i in range(5):
plot_and_train(k4(dfs['repo'].unique()[i]))
#### REQ8-5). Plot the closed issues forecast
In [ ]:
def k5(reponame):
df = dfs6[dfs6['repo'] ==reponame]
df = df.groupby('closed_at')['closed_at']
df_new = df.describe()
s1 = pd.Series(df_new['top'], name='ds_orig')
df_new = pd.concat([df_new, s1], axis=1)
pdf = pd.DataFrame(['ds','ds_orig','y'])
pdf = df_new[['top','ds_orig','count']]
pdf.columns = ['ds','ds_orig','y']
pdf['ds_new'] = pd.to_datetime(pdf['ds'])
df_final = pd.DataFrame(['ds','y'])
df_final = pdf[['ds_orig','y']]
df_final.columns = ['timestamp','value']
firstDay = min(pd.to_datetime(df_final['timestamp']))
df_final['time'] = [float(x.days) for x in \
[x - firstDay for x in pd.to_datetime(df_final['timestamp'])]]
return df_final
In [487]:
for i in range(5):
plot_and_train(k5(dfs['repo'].unique()[i]))
#### REQ8-6). Plot the pulls forecast
In [ ]:
#Add your code for requirement 8.6 in this cell
#### REQ8-7). Plot the commits forecast
In [ ]:
#Add your code for requirement 8.7 in this cell
#### REQ8-8). Plot the branches forecast
In [ ]:
#Add your code for requirement 8.8 in this cell
#### REQ8-9). Plot the contributors forecast
In [ ]:
#Add your code for requirement 8.9 in this cell
#### REQ8-10). Plot the releases forecast
In [ ]:
#Add your code for requirement 8.10 in this cell
## REQUIREMENT 8 (STATSMODEL)
- Use StatsModel package to forecast the following for every repository
In [ ]:
import statsmodels.api as sm
In [ ]:
def plot_arima(df):
model = sm.tsa.ARIMA(np.asarray(df['value']),order=(1,0,0))
results = model.fit()
df['forecast'] = results.fittedvalues
df[['value','forecast']].plot(figsize=(16,12))
#### REQ8-1). The day of the week maximum number of issues created
In [ ]:
def s1(reponame):
df = dfs[dfs['repo'] ==reponame]
df = df.groupby('created_at')['created_at']
df_new = df.describe()
dfnew1 = pd.Series(df_new['top'], name='ds_original')
df_new = pd.concat([df_new, dfnew1], axis=1)
datafrm_pdf = pd.DataFrame(['ds','ds_original','y'])
datafrm_pdf = df_new[['top','ds_original','count']]
datafrm_pdf.columns = ['ds','ds_original','y']
datafrm_pdf['ds_new'] = pd.to_datetime(datafrm_pdf['ds']) - pd.to_timedelta(7, unit='d')
df_weekly_maximum = datafrm_pdf.reset_index().groupby([pd.Grouper(key='ds_new', freq='W-MON')]).apply(lambda x: x.loc[x.y == x.y.max(),['ds_original','y']])
df_created_output = pd.DataFrame(['ds','y'])
df_created_output = df_weekly_maximum[['ds_original','y']]
df_created_output.columns = ['ds','y']
tensor_Created = df_created_output
tensor_Created = tensor_Created[['ds','y']]
df = pd.DataFrame(tensor_Created)
tensor_Created.rename(columns={'ds':'timestamp'}, inplace=True)
tensor_Created.rename(columns={'y':'value'}, inplace=True)
# print(tensor_Created)
firstDay = min(pd.to_datetime(tensor_Created['timestamp']))
tensor_Created['time'] = [float(x.days) for x in \
[x - firstDay for x in pd.to_datetime(tensor_Created['timestamp'])]]
return tensor_Created
In [489]:
for i in range(5):
plot_arima(s1(dfs['repo'].unique()[i]))
#### REQ8-2). The day of the week maximum number of issues closed
In [491]:
for i in range(5):
plot_arima(k2(dfs['repo'].unique()[i]))
#### REQ8-3). The month of the year that has maximum number of issues closed
In [492]:
for i in range(5):
plot_arima(k3(dfs['repo'].unique()[i]))
#### REQ8-4). Plot the created issues forecast
In [493]:
for i in range(5):
plot_arima(k4(dfs['repo'].unique()[i]))
#### REQ8-5). Plot the closed issues forecast
In [494]:
for i in range(5):
plot_arima(k5(dfs['repo'].unique()[i]))
#### REQ8-6). Plot the pulls forecast
In [ ]:
#Add your code for requirement 8.6 in this cell
#### REQ8-7). Plot the commits forecast
In [ ]:
#Add your code for requirement 8.7 in this cell
#### REQ8-8). Plot the branches forecast
In [ ]:
#Add your code for requirement 8.8 in this cell
#### REQ8-9). Plot the contributors forecast
In [ ]:
#Add your code for requirement 8.9 in this cell
#### REQ8-10). Plot the releases forecast
In [ ]:
#Add your code for requirement 8.10 in this cell